import scrapy
from w3lib.html import remove_tags
import re
class DoubGroupSpider(scrapy.Spider):

    name = 'dbgroup'
    custom_settings = {
        "DEFAULT_REQUEST_HEADERS": {
            "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
            # 'Host': 'https://www.douban.com',
        },
    }
    def start_requests(self):
        for i in range(0,20,20):
            base_url = 'https://www.douban.com/group/search?start={page}&cat=1019&sort=relevance&q=%E6%97%B6%E5%B0%9A'.format(page=i)
            yield scrapy.Request(base_url,callback=self.parse_list)
    def parse_list(self,response):

        html_response = response.xpath('//div[@class="groups"]//div[@class="result"]')

        for html in html_response:

            group_url = html.xpath('.//div[@class="title"]/h3/a/@href').extract()[0]
            group_name = html.xpath('.//div[@class="title"]/h3/a/text()').extract()[0]
            group_info = html.xpath('.//div[@class="info"]/text()').extract()[0]
            group_describ = html.xpath('.//div/p/text()').extract()[0]

            # print(group_url,group_name,group_info,group_describ)
            yield scrapy.Request(group_url,callback=self.parse_info)
    def parse_info(self, response):
        group_board = response.xpath('//div[@class="article"]/div[@class="group-board"]')
        group_creat = group_board.xpath('./p/text()')[0].extract().replace('\n','').replace(' ','') #创建时间
        if group_board.xpath('./p/a/text()'):
            group_leader = group_board.xpath('./p/a/text()')[0].extract()
            group_info = group_creat + group_leader
        group_info = group_creat
        if group_board.xpath('./p/a/@href'):
            group_leader_url = group_board.xpath('./p/a/@href')[0].extract() # 组长详情，唯一标识
        group_leader_url = ''
        group_intro = group_board.xpath('./div[@class="group-intro"]/text()').extract()
        group_describ = [x.replace('\r','').replace('\n','').replace(' ','') for x in group_intro] # 去除非法字符串
        group_list = ''.join(group_describ)
        print(group_list)
        # group_describe = remove_tags(group_intro[0])
        if group_board.xpath('./div[@class="group-tags"]'):
            group_t = group_board.xpath('./div[@class="group-tags"]//a[@class="tag"]/text()').extract()
            group_tags = ','.join(group_t)

